﻿using Spider.Framework.UtilityCommon;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using Spider.Framework.UtilityCommon.Extensions;
using System.Net;
using System.Text;
using System.IO;
using System;
using MyCrawler.Entity;
using MyCrawler.Utilyties;
using Spider.Framework.Download;

namespace MyCrawler.Controls
{
    public class VnExpressCtr : ForumsCtr
    {
        public VnExpressCtr()
        {
        }

        public void GetListCatagories()
        {
            try
            {
                string html = GetContentWithStream("http://vnexpress.net/", -1);

                if (html != null || html != "")
                {
                    html = UtilityCtr.GetItemFirst(html, @"(<ul[\s]+id='menu_web').*?(<\/ul>)");

                    List<string> lisLi = UtilityCtr.GetListItem(html, @"(<li).*?(<\/li>)");

                    for (int i = 0; i < lisLi.Count; i++)
                    {
                        string lnkCat = UtilityCtr.GetItemFirst(lisLi[i], @"(href=').*?(')").Replace("href='", "").Replace("'", "");
                        if (lnkCat.Contains("http://vnexpress.net") == false && lnkCat.Contains("tin-tuc/thoi-su") == false && lnkCat.Contains("tin-tuc/goc-nhin") == false)
                        {
                            if (!lnkCat.Contains(".vnexpress.net"))
                            {
                                GetActicleInCatagories("http://vnexpress.net" + lnkCat); // không phải sub domain
                            }
                            else
                            {
                                if (lnkCat.Contains("http://kinhdoanh.vnexpress.net"))
                                {
                                    GetActicleInCatagories("http://kinhdoanh.vnexpress.net");
                                }
                            }
                        }
                    }
                }
            }
            catch
            { }
        }

        private void GetActicleInCatagories(string lnk)
        {
            try
            {
                string html = GetContentWithStream(lnk, -1);

                if (html != null || html != "")
                {
                    GetArticles(html);

                    // kiểm tra nút next còn tồn tại không
                    bool isNext = Regex.IsMatch(html, @"pagination_next.gif");

                    if (isNext == true)
                    {
                        // Nếu có phân trang
                        for (int i = 2; i <= 2000; i++) // chạy đến trang thứ 1000 thì thôi
                        {
                            string lnkPageNext = lnk + "/page/" + i + ".html";

                            html = GetContentWithStream(lnkPageNext, -1);

                            isNext = Regex.IsMatch(html, @"pagination_next.gif");

                            GetArticles(html);

                            if (isNext == false)
                            {
                                break;
                            }
                        }
                    }
                }
            }
            catch
            { }
        }

        private void GetArticles(string html)
        {
            try
            {
                string htmlUl = UtilityCtr.GetItemFirst(html, @"(id='news_home'>)(.*?)(<\/ul>[\s]+<div class='bottom_pagination[\s]+width_common'>)");

                List<string> lisCmt = new List<string>();

                List<string> lisArticle = UtilityCtr.GetListItem(htmlUl, @"(<div class='title_news'>).*?(<\/div>)");
                for (int i = 0; i < lisArticle.Count; i++)
                {
                    string lnkArt = UtilityCtr.GetItemFirst(lisArticle[i], @"(href=')(.*?)(')").Replace("href='", "").Replace("'", "");

                    string subj = UtilityCtr.GetItemFirst(lisArticle[i], @"(<a)(.*?)(<\/a>)");
                    subj = Regex.Replace(subj, "<.*?>", "").Trim();

                    string sId = UtilityCtr.GetItemFirst(lnkArt, @"-[\d]+.html").Replace("-", "").Replace(".html", "");
                    int id;
                    try
                    {
                        id = Convert.ToInt32(sId);
                    }
                    catch
                    {
                        id = 0;
                    }

                    lisCmt = GetComments(lnkArt);

                    string htmlContent = GetContentWithStream(lnkArt, -1);
                    string cont = UtilityCtr.GetItemFirst(htmlContent, @"(<div[\s]+id='left_calculator'>)(.*?)(<div[\s]+class='social_share width_common'[\s]+id='social_like'.*?>)");

                    if (cont != "")
                    {
                        cont = Regex.Replace(cont, @"<div class='block_timer_share'>.*?(fck_detail width_common'>)|(<script)(.*?)(<\/script>)", "");
                        cont = Regex.Replace(cont, @"<.*?>", " ");
                        cont = Regex.Replace(cont, @"[\s]+", " ").Trim();
                    }

                    string htmlCat = UtilityCtr.GetItemFirst(htmlContent, @"(<ul class='list_arrow_breakumb'>)(.*?)(<\/ul>)");
                    string catName = UtilityCtr.GetItemFirst(htmlCat, @"<a.*?<\/a>");
                    catName = Regex.Replace(catName, @"<.*?>", " ");
                    catName = Regex.Replace(catName, @"[\s]+", " ").Trim();

                    string datePost = UtilityCtr.GetItemFirst(htmlContent, @"(<div[\s]+class='block_timer_share'>[\s]+<div[\s]+class='block_timer[\s]+left[\s]+txt_666'>)(.*?)(<\/div>)");
                    string date = UtilityCtr.GetItemFirst(datePost, @"[\d]+\/[\d]+\/[\d]+");
                    string[] arrDate = date.Split('/');
                    if (arrDate.Length > 0)
                    {
                        date = arrDate[2] + "-" + arrDate[1] + "-" + arrDate[0];
                    }

                    string time = UtilityCtr.GetItemFirst(datePost, @"[\d]+:[\d]+");

                    DateTime lastModify;
                    try
                    {
                        lastModify = Convert.ToDateTime(date + " " + time);
                    }
                    catch
                    {
                        lastModify = DateTime.Now;
                    }
                    #region [Ghi nhận]
                    ContentEntity entity = new ContentEntity();

                    entity.Category = catName;
                    entity.Comments = lisCmt;
                    entity.Content = cont;
                    entity.Id = id;
                    entity.LastModify = lastModify;
                    entity.Link = lnkArt;
                    entity.Source = "vnexpress.net";
                    entity.Summury = "";
                    entity.Title = subj;

                    // ghi file json
                    string file = SaveFileJson(entity);
                    entity.File = file;

                    // ghi xuống database

                    ContentBO conBo = new ContentBO();
                    conBo.AddNew(entity); // thêm mới

                    #endregion
                }
            }
            catch
            { }
        }

        /// <summary>
        /// Lấy danh sách comment theo url của bài viết
        /// </summary>
        /// <param name="url"></param>
        /// <returns>trả ra List<string>Chứa danh sách các bình luận</returns>
        public List<string> GetComments(string url)
        {
            try
            {
                string objId = url.GetElement(@"[\d]+.html", false, PatternType.Regex).Replace(".html", "");

                string srcJson = "";
                string content = "";
                string siteId = "";
                string urlFromVnExpress = "http://interactions.vnexpress.net/index/get?offset=0&limit=1000&objectid=" + objId;

                #region [trang gốc]
                if (url.Contains("http://vnexpress.net")) // trang gốc
                {
                    siteId = "1000000";
                    srcJson = urlFromVnExpress + "&objecttype=1&siteid=" + siteId;

                    content = GetContentWithStream(srcJson);
                    if (content.Length < 150) // nếu không có dữ liệu cố thử với objecttype=3
                    {
                        srcJson = urlFromVnExpress + "&objecttype=3&siteid=" + siteId;
                        content = GetContentWithStream(srcJson);
                    }

                    goto LabelContinue;
                }
                #endregion

                #region [chuyên mục thể thao]
                if (url.Contains("thethao.vnexpress.net"))
                {
                    siteId = "1002565";
                    srcJson = urlFromVnExpress + "&objecttype=1&siteid=" + siteId;

                    content = GetContentWithStream(srcJson);
                    if (content.Length < 150)  // nếu không có dữ liệu cố thử với objecttype=3
                    {
                        srcJson = urlFromVnExpress + "&objecttype=3&siteid=" + siteId;
                        content = GetContentWithStream(srcJson);
                    }

                    goto LabelContinue;
                }
                #endregion

                #region [chuyên mục kinh doanh]
                if (url.Contains("kinhdoanh.vnexpress.net")) // chuyên mục kinh doanh
                {
                    siteId = "1003159";
                    srcJson = urlFromVnExpress + "&objecttype=1&siteid=" + siteId;

                    content = GetContentWithStream(srcJson);
                    if (content.Length < 150)  // nếu không có dữ liệu cố thử với objecttype=3
                    {
                        srcJson = urlFromVnExpress + "&objecttype=3&siteid=" + siteId;
                        content = GetContentWithStream(srcJson);
                    }

                    goto LabelContinue;
                }
                #endregion

                #region [chuyên mục đời sống]
                if (url.Contains("doisong.vnexpress.net"))
                {
                    siteId = "1002966";
                    srcJson = urlFromVnExpress + "&objecttype=3&siteid=" + siteId;

                    content = GetContentWithStream(srcJson);
                    if (content.Length < 150)  // nếu không có dữ liệu cố thử với objecttype=1
                    {
                        srcJson = urlFromVnExpress + "&objecttype=1&siteid=" + siteId;
                        content = GetContentWithStream(srcJson);
                    }

                    goto LabelContinue;
                }
                #endregion

                #region [chuyên mục giải trí]
                if (url.Contains("giaitri.vnexpress.net"))
                {
                    siteId = "1002691";
                    srcJson = urlFromVnExpress + "&objecttype=1&siteid=" + siteId;

                    content = GetContentWithStream(srcJson);
                    if (content.Length < 150)  // nếu không có dữ liệu cố thử với objecttype=3
                    {
                        srcJson = urlFromVnExpress + "&objecttype=3&siteid=" + siteId;
                        content = GetContentWithStream(srcJson);
                    }

                    goto LabelContinue;
                }
                #endregion

                #region [chuyên mục du lịch]
                if (url.Contains("dulich.vnexpress.net"))
                {
                    siteId = "1003231";
                    srcJson = urlFromVnExpress + "&objecttype=1&siteid=" + siteId;
                    content = GetContentWithStream(srcJson);
                    if (content.Length < 150)  // nếu không có dữ liệu cố thử với objecttype=3
                    {
                        srcJson = urlFromVnExpress + "&objecttype=3&siteid=" + siteId;
                        content = GetContentWithStream(srcJson);
                    }

                    goto LabelContinue;
                }
                #endregion

                #region [chuyên mục số hóa]
                if (url.Contains("sohoa.vnexpress.net"))
                {
                    siteId = "1002592";
                    srcJson = urlFromVnExpress + "&objecttype=1&siteid=" + siteId;

                    content = GetContentWithStream(srcJson);
                    if (content.Length < 150)  // nếu không có dữ liệu cố thử với objecttype=3
                    {
                        srcJson = urlFromVnExpress + "&objecttype=3&siteid=1002592" + siteId;
                        content = GetContentWithStream(srcJson);
                    }

                    goto LabelContinue;
                }
                #endregion

                #region [lấy bình luận]
            LabelContinue:

                List<string> ls = new List<string>();

                if (content != "")
                {
                    content = ConvertANSIToUnicode(content).Replace('\"', '"'); // định dạng lại nếu có mã ansi
                    content = ConvertNCRDecimalToUnicode(content); // nếu có mã NCR decimal thì chuyển sang unicode

                    string patt = "(comment_id\\\":)(.*?)(}})";
                    var lisCmt = content.GetElements(patt, false, PatternType.Regex);
                    if (lisCmt == null || lisCmt.Count == 0)
                    {
                        return null;
                    }

                    for (int i = 0; i < lisCmt.Count; i++)
                    {
                        string pattContent = "(content\\\":\\\").*?(\\\",)";
                        var arrContent = lisCmt[i].GetElements(pattContent, false, PatternType.Regex);

                        for (int j = 0; j < arrContent.Count; j++)
                        {
                            string cmt = Regex.Replace(arrContent[j], "(<.*?>)|(content\\\":\\\")|(\\\",)", "");
                            ls.Add(cmt.Trim());
                        }

                        // làm việc với các reply nếu có
                        string sTotal = lisCmt[i].GetElement("\\\"total\\\":[\\d]+", false, PatternType.Regex);
                        try
                        {
                            int ttReply = Convert.ToInt32(sTotal.GetElement(@"[\d]+", false, PatternType.Regex));

                            if (ttReply > 2) // có nhiều hơn 2 bình luận
                            {
                                try
                                {
                                    int idCmt = Convert.ToInt32(lisCmt[i].GetElement(@"[\d]+", false, PatternType.Regex));
                                    string linkReply = "http://interactions.vnexpress.net//index/getreplay?siteid=" + siteId + "&objectid=" + objId + "&objecttype=1&id=" + idCmt.ToString() + "&limit=1000";
                                    string reply = GetContentWithStream(linkReply);

                                    // nếu không có dữ liệu cố thử với objecttype=3 xem có data không?
                                    if (reply.Length < 150)
                                    {
                                        linkReply = "http://interactions.vnexpress.net//index/getreplay?siteid=" + siteId + "&objectid=" + objId + "&objecttype=3&id=" + idCmt.ToString() + "&limit=1000";
                                        reply = GetContentWithStream(linkReply);
                                    }

                                    reply = ConvertANSIToUnicode(reply).Replace('\"', '"'); // định dạng lại nếu có mã ansi
                                    reply = ConvertNCRDecimalToUnicode(reply); // nếu có mã NCR decimal thì chuyển sang unicode

                                    string pattRep = "(content\\\":\\\").*?(\",)";
                                    var arrRep = reply.GetElements(pattRep, false, PatternType.Regex);

                                    for (int k = 2; k < arrRep.Count; k++) // đi từ reply thứ 3 vì 2 item đã lấy ở trên
                                    {
                                        string item = Regex.Replace(arrRep[k], "(<.*?>)|(content\\\":\\\")|(\\\",)", ""); // làm sạch code html nếu có
                                        ls.Add(item.Trim());
                                    }
                                }
                                catch { }
                            }
                        }
                        catch { }
                    }
                }
                #endregion

                return ls;
            }
            catch
            {
                return null;
            }
        }

        /// <summary>
        /// chuyển đổi mã Ansi tới unicode
        /// </summary>
        /// <param name="content"></param>
        /// <returns></returns>
        public string ConvertANSIToUnicode(string content)
        {
            bool hasAnsi = Regex.IsMatch(content, @"\\u[\d]+");

            if (hasAnsi == false)
            {
                return content;
            }

            string s = content.Replace("\\u00e0", "à");
            s = s.Replace("\\u00e1", "á");
            s = s.Replace("\\u1ea3", "ả");
            s = s.Replace("\\u00e3", "ã");
            s = s.Replace("\\u1ea1", "ạ");
            s = s.Replace("\\u0103", "ă");
            s = s.Replace("\\u1eb1", "ằ");
            s = s.Replace("\\u1eaf", "ắ");
            s = s.Replace("\\u1eb3", "ẳ");
            s = s.Replace("\\u1eb5", "ẵ");
            s = s.Replace("\\u1eb7", "ặ");
            s = s.Replace("\\u00e2", "â");
            s = s.Replace("\\u1ea7", "ầ");
            s = s.Replace("\\u1ea5", "ấ");
            s = s.Replace("\\u1ea9", "ẩ");
            s = s.Replace("\\u1eab", "ẫ");
            s = s.Replace("\\u1ead", "ậ");

            s = s.Replace("\\u0111", "đ");
            s = s.Replace("\\u1ead", "ậ");
            s = s.Replace("\\u1ead", "ậ");
            s = s.Replace("\\u1ead", "ậ");
            s = s.Replace("\\u1ead", "ậ");


            s = s.Replace("\\u00e8", "è");
            s = s.Replace("\\u00e9", "é");
            s = s.Replace("\\u1ebb", "ẻ");
            s = s.Replace("\\u1ebd", "ẽ");
            s = s.Replace("\\u1eb9", "ẹ");
            s = s.Replace("\\u00ea", "ê");
            s = s.Replace("\\u1ec1", "ề");
            s = s.Replace("\\u1ebf", "ế");
            s = s.Replace("\\u1ec3", "ể");
            s = s.Replace("\\u1ec5", "ễ");
            s = s.Replace("\\u1ec7", "ệ");

            s = s.Replace("\\u00f2", "ò");
            s = s.Replace("\\u00f3", "ó");
            s = s.Replace("\\u1ecf", "ỏ");
            s = s.Replace("\\u00f5", "õ");
            s = s.Replace("\\u1ecd", "ọ");
            s = s.Replace("\\u00f4", "ô");
            s = s.Replace("\\u1ed3", "ồ");
            s = s.Replace("\\u1ed1", "ố");
            s = s.Replace("\\u1ed5", "ổ");
            s = s.Replace("\\u1ed7", "ỗ");
            s = s.Replace("\\u1ed9", "ộ");
            s = s.Replace("\\u01a1", "ơ");
            s = s.Replace("\\u1edd", "ờ");
            s = s.Replace("\\u1edb", "ớ");
            s = s.Replace("\\u1edf", "ở");
            s = s.Replace("\\u1ee1", "ỡ");
            s = s.Replace("\\u1ee3", "ợ");

            s = s.Replace("\\u00f9", "ù");
            s = s.Replace("\\u00fa", "ú");
            s = s.Replace("\\u1ee7", "ủ");
            s = s.Replace("\\u0169", "ũ");
            s = s.Replace("\\u1ee5", "ụ");
            s = s.Replace("\\u01b0", "ư");
            s = s.Replace("\\u1eeb", "ừ");
            s = s.Replace("\\u1ee9", "ứ");
            s = s.Replace("\\u1eed", "ử");
            s = s.Replace("\\u1eef", "ữ");
            s = s.Replace("\\u1ef1", "ự");

            s = s.Replace("\\u00c0", "À");
            s = s.Replace("\\u00c1", "Á");
            s = s.Replace("\\u1ea2", "Ả");
            s = s.Replace("\\u00c3", "Ã");
            s = s.Replace("\\u1ea0", "Ạ");
            s = s.Replace("\\u0102", "Â");
            s = s.Replace("\\u1eb0", "Ầ");
            s = s.Replace("\\u1eae", "Ấ");
            s = s.Replace("\\u1eb2", "Ẩ");
            s = s.Replace("\\u1eb4", "Ẫ");
            s = s.Replace("\\u1eb6", "Ậ");
            s = s.Replace("\\u00c2", "Ă");
            s = s.Replace("\\u1ea6", "Ằ");
            s = s.Replace("\\u1ea4", "Ắ");
            s = s.Replace("\\u1ea8", "Ẳ");
            s = s.Replace("\\u1eaa", "Ẵ");
            s = s.Replace("\\u1eac", "Ặ");

            s = s.Replace("\\u0110", "Đ");

            s = s.Replace("\\u00c8", "È");
            s = s.Replace("\\u00c9", "É");
            s = s.Replace("\\u1eba", "Ẻ");
            s = s.Replace("\\u1ebc", "Ẽ");
            s = s.Replace("\\u1eb8", "Ẹ");
            s = s.Replace("\\u00ca", "Ê");
            s = s.Replace("\\u1ec0", "Ề");
            s = s.Replace("\\u1ebe", "Ế");
            s = s.Replace("\\u1ec2", "Ể");
            s = s.Replace("\\u1ec4", "Ễ");
            s = s.Replace("\\u1ec6", "Ệ");

            s = s.Replace("\\u00d2", "Ò");
            s = s.Replace("\\u00d3", "Ó");
            s = s.Replace("\\u1ece", "Ỏ");
            s = s.Replace("\\u00d5", "Õ");
            s = s.Replace("\\u1ec6", "Ọ");
            s = s.Replace("\\u00d4", "Ô");
            s = s.Replace("\\u1ed2", "Ồ");
            s = s.Replace("\\u1ed0", "Ố");
            s = s.Replace("\\u1ed4", "Ổ");
            s = s.Replace("\\u1ed6", "Ỗ");
            s = s.Replace("\\u1ed8", "Ộ");
            s = s.Replace("\\u01a0", "Ơ");
            s = s.Replace("\\u1edc", "Ờ");
            s = s.Replace("\\u1eda", "Ớ");
            s = s.Replace("\\u1ede", "Ở");
            s = s.Replace("\\u1ee0", "Ỡ");
            s = s.Replace("\\u1ee2", "Ợ");

            s = s.Replace("\\u00d9", "Ù");
            s = s.Replace("\\u00da", "Ú");
            s = s.Replace("\\u1ee6", "Ủ");
            s = s.Replace("\\u0168", "Ũ");
            s = s.Replace("\\u1ee4", "Ụ");
            s = s.Replace("\\u01af", "Ư");
            s = s.Replace("\\u1eea", "Ừ");
            s = s.Replace("\\u1ee8", "Ứ");
            s = s.Replace("\\u1eec", "Ử");
            s = s.Replace("\\u1eee", "Ữ");
            s = s.Replace("\\u1ef0", "Ự");

            s = s.Replace("\\u00ec", "ì");
            s = s.Replace("\\u00ed", "í");
            s = s.Replace("\\u1ec9", "ỉ");
            s = s.Replace("\\u0129", "ĩ");
            s = s.Replace("\\u1ecb", "ị");
            s = s.Replace("\\u00cc", "Ì");
            s = s.Replace("\\u00cd", "Í");
            s = s.Replace("\\u1ec8", "Ỉ");
            s = s.Replace("\\u0128", "Ĩ");
            s = s.Replace("\\u1eca", "Ị");

            s = s.Replace("\\u1ef3", "ỳ");
            s = s.Replace("\\u00fd", "ý");
            s = s.Replace("\\u1ef7", "ỷ");
            s = s.Replace("\\u1ef9", "ỹ");
            s = s.Replace("\\u1ef5", "ỵ");
            s = s.Replace("\\u1ef2", "Ỳ");
            s = s.Replace("\\u00dd", "Ý");
            s = s.Replace("\\u1ef6", "Ỷ");
            s = s.Replace("\\u1ef8", "Ỹ:");
            s = s.Replace("\\u1ef4", "Ỵ");

            #region [dấu sắc]
            s = s.Replace("a\\u0301", "á");
            s = s.Replace("ă\\u0301", "ắ");
            s = s.Replace("â\\u0301", "ấ");
            s = s.Replace("e\\u0301", "é");
            s = s.Replace("ê\\u0301", "ế");
            s = s.Replace("i\\u0301", "í");
            s = s.Replace("y\\u0301", "ý");
            s = s.Replace("o\\u0301", "ó");
            s = s.Replace("ô\\u0301", "ố");
            s = s.Replace("u\\u0301", "ú");
            s = s.Replace("ư\\u0301", "ứ");

            s = s.Replace("A\\u0301", "Á");
            s = s.Replace("Ă\\u0301", "Ắ");
            s = s.Replace("Â\\u0301", "Ấ");
            s = s.Replace("E\\u0301", "É");
            s = s.Replace("Ê\\u0301", "Ế");
            s = s.Replace("I\\u0301", "Í");
            s = s.Replace("Y\\u0301", "Ý");
            s = s.Replace("O\\u0301", "Ó");
            s = s.Replace("Ô\\u0301", "Ố");
            s = s.Replace("U\\u0301", "Ú");
            s = s.Replace("Ư\\u0301", "Ứ");
            #endregion

            #region [dấu huyền]
            s = s.Replace("a\\u0300", "à");
            s = s.Replace("ă\\u0300", "ằ");
            s = s.Replace("â\\u0300", "ầ");
            s = s.Replace("e\\u0300", "è");
            s = s.Replace("ê\\u0300", "ề");
            s = s.Replace("i\\u0300", "ì");
            s = s.Replace("y\\u0300", "ỳ");
            s = s.Replace("o\\u0300", "ò");
            s = s.Replace("ô\\u0300", "ồ");
            s = s.Replace("ơ\\u0300", "ờ");
            s = s.Replace("u\\u0300", "ù");
            s = s.Replace("ư\\u0300", "ừ");

            s = s.Replace("A\\u0300", "À");
            s = s.Replace("Ă\\u0300", "Ằ");
            s = s.Replace("Â\\u0300", "Ầ");
            s = s.Replace("E\\u0300", "È");
            s = s.Replace("Ê\\u0300", "Ề");
            s = s.Replace("I\\u0300", "Ì");
            s = s.Replace("Y\\u0300", "Ỳ");
            s = s.Replace("O\\u0300", "Ò");
            s = s.Replace("Ô\\u0300", "Ồ");
            s = s.Replace("Ơ\\u0300", "Ờ");
            s = s.Replace("U\\u0300", "Ù");
            s = s.Replace("Ư\\u0300", "Ừ");
            #endregion

            #region [dấu hoi]
            s = s.Replace("a\\u0309", "ả");
            s = s.Replace("ă\\u0309", "ẳ");
            s = s.Replace("â\\u0309", "ẩ");
            s = s.Replace("e\\u0309", "ẻ");
            s = s.Replace("ê\\u0309", "ể");
            s = s.Replace("i\\u0309", "ỉ");
            s = s.Replace("y\\u0309", "ỷ");
            s = s.Replace("o\\u0309", "ỏ");
            s = s.Replace("ô\\u0309", "ổ");
            s = s.Replace("u\\u0309", "ủ");
            s = s.Replace("ư\\u0309", "ử");

            s = s.Replace("A\\u0309", "Ả");
            s = s.Replace("Ă\\u0309", "Ẳ");
            s = s.Replace("Â\\u0309", "Ẩ");
            s = s.Replace("E\\u0309", "Ể");
            s = s.Replace("Ê\\u0309", "Ể");
            s = s.Replace("I\\u0309", "Ỉ");
            s = s.Replace("Y\\u0309", "Ỷ");
            s = s.Replace("O\\u0309", "Ỏ");
            s = s.Replace("Ô\\u0309", "Ổ");
            s = s.Replace("U\\u0309", "Ủ");
            s = s.Replace("Ư\\u0309", "Ử");
            #endregion

            #region [dấu ngã]
            s = s.Replace("a\\u0303", "ã");
            s = s.Replace("ă\\u0303", "ẵ");
            s = s.Replace("â\\u0303", "ẫ");
            s = s.Replace("e\\u0303", "ẽ");
            s = s.Replace("ê\\u0303", "ễ");
            s = s.Replace("i\\u0303", "ĩ");
            s = s.Replace("y\\u0303", "ỹ");
            s = s.Replace("o\\u0303", "õ");
            s = s.Replace("ô\\u0303", "ỗ");
            s = s.Replace("u\\u0303", "ũ");
            s = s.Replace("ư\\u0303", "ữ");

            s = s.Replace("A\\u0303", "Ã");
            s = s.Replace("Ă\\u0303", "Ẵ");
            s = s.Replace("Â\\u0303", "Ẫ");
            s = s.Replace("E\\u0303", "Ẽ");
            s = s.Replace("Ê\\u0303", "Ễ");
            s = s.Replace("I\\u0303", "Ĩ");
            s = s.Replace("Y\\u0303", "Ỹ");
            s = s.Replace("O\\u0303", "Õ");
            s = s.Replace("Ô\\u0303", "Ỗ");
            s = s.Replace("U\\u0303", "Ũ");
            s = s.Replace("Ư\\u0303", "Ữ");
            #endregion

            #region [dấu nặng]
            s = s.Replace("a\\u0323", "ạ");
            s = s.Replace("ă\\u0323", "ặ");
            s = s.Replace("â\\u0323", "ậ");
            s = s.Replace("e\\u0323", "ẹ");
            s = s.Replace("ê\\u0323", "ệ");
            s = s.Replace("i\\u0323", "ị");
            s = s.Replace("y\\u0323", "ỵ");
            s = s.Replace("o\\u0323", "ọ");
            s = s.Replace("ô\\u0323", "ộ");
            s = s.Replace("u\\u0323", "ụ");
            s = s.Replace("ư\\u0323", "ự");

            s = s.Replace("A\\u0323", "Ạ");
            s = s.Replace("Ă\\u0323", "Ặ");
            s = s.Replace("Â\\u0323", "Ậ");
            s = s.Replace("E\\u0323", "Ẹ");
            s = s.Replace("Ê\\u0323", "Ệ");
            s = s.Replace("I\\u0323", "Ị");
            s = s.Replace("Y\\u0323", "Ỵ");
            s = s.Replace("O\\u0323", "Ọ");
            s = s.Replace("Ô\\u0323", "Ộ");
            s = s.Replace("U\\u0323", "Ụ");
            s = s.Replace("Ư\\u0323", "Ự");
            #endregion

            #region [Khoảng trắng]
            s = s.Replace("&nbsp;", " ");
            #endregion
            return s;
        }

        public string ConvertNCRDecimalToUnicode(string content)
        {
            // kiểm tra chuỗi có ký tự NCRDecimal ko?
            bool hasNcr = Regex.IsMatch(content, @"#[\d]+;");

            if (hasNcr == false)
            {
                return content;
            }

            string s = content.Replace("&#97;", "a");
            s = s.Replace("&#225;", "á");
            s = s.Replace("&#224;", "à");
            s = s.Replace("&#7843;", "ả");
            s = s.Replace("&#227;", "ã");
            s = s.Replace("&#7841;", "ạ");

            s = s.Replace("&#259;", "ă");
            s = s.Replace("&#7855;", "ắ");
            s = s.Replace("&#7857;", "ằ");
            s = s.Replace("&#7859;", "ẳ");
            s = s.Replace("&#7861;", "ẵ");
            s = s.Replace("&#7863;", "ặ");

            s = s.Replace("&#226;", "â");
            s = s.Replace("&#7845;", "ấ");
            s = s.Replace("&#7847;", "ầ");
            s = s.Replace("&#7849;", "ẩ");
            s = s.Replace("&#7851;", "ẵ");
            s = s.Replace("&#7853;", "ậ");

            s = s.Replace("&#101;", "e");
            s = s.Replace("&#233;", "é");
            s = s.Replace("&#232;", "è");
            s = s.Replace("&#7867;", "ẻ");
            s = s.Replace("&#7869;", "ẽ");
            s = s.Replace("&#7865;", "ẹ");

            s = s.Replace("&#234;", "ê");
            s = s.Replace("&#7871;", "ế");
            s = s.Replace("&#7873;", "ề");
            s = s.Replace("&#7875;", "ể");
            s = s.Replace("&#7877;", "ễ");
            s = s.Replace("&#7879;", "ệ");

            s = s.Replace("&#105;", "i");
            s = s.Replace("&#237;", "í");
            s = s.Replace("&#236;", "ì");
            s = s.Replace("&#7881;", "ỉ");
            s = s.Replace("&#297;", "ĩ");
            s = s.Replace("&#7883;", "ị");

            s = s.Replace("&#111;", "o");
            s = s.Replace("&#243;", "ó");
            s = s.Replace("&#242;", "ò");
            s = s.Replace("&#7887;", "ỏ");
            s = s.Replace("&#245;", "õ");
            s = s.Replace("&#7885;", "ọ");

            s = s.Replace("&#244;	", "ô");
            s = s.Replace("&#7889;", "ố");
            s = s.Replace("&#7891;", "ồ");
            s = s.Replace("&#7893;", "ổ");
            s = s.Replace("&#7895;", "ỗ");
            s = s.Replace("&#7897;", "ộ");

            s = s.Replace("&#417;", "ơ");
            s = s.Replace("&#7899;", "ớ");
            s = s.Replace("&#7901;", "ờ");
            s = s.Replace("&#7903;", "ở");
            s = s.Replace("&#7905;", "ỡ");
            s = s.Replace("&#7907;", "ợ");

            s = s.Replace("&#117;", "u");
            s = s.Replace("&#250;", "ú");
            s = s.Replace("&#249;", "ù");
            s = s.Replace("&#7911;", "ủ");
            s = s.Replace("&#361;", "ũ");
            s = s.Replace("&#7909;", "ụ");

            s = s.Replace("&#432;", "y");
            s = s.Replace("&#253;", "ý");
            s = s.Replace("&#7923;", "ỳ");
            s = s.Replace("&#7927;", "ỷ");
            s = s.Replace("&#7927;", "ỹ");
            s = s.Replace("&#7925;", "ỵ");

            s = s.Replace("&#273;", "đ");

            s = s.Replace("&#65;", "A");
            s = s.Replace("&#193;", "Á");
            s = s.Replace("&#192;", "À");
            s = s.Replace("&#7842;", "Ả");
            s = s.Replace("&#195;", "Ã");
            s = s.Replace("&#7840;", "Ạ");

            s = s.Replace("&#258;", "Ă");
            s = s.Replace("&#7854;", "Ắ");
            s = s.Replace("&#7856;", "Ằ");
            s = s.Replace("&#7858;", "Ẳ");
            s = s.Replace("&#7860;", "Ẵ");
            s = s.Replace("&#7862;", "Ặ");

            s = s.Replace("&#194;", "Â");
            s = s.Replace("&#7844;", "Ấ");
            s = s.Replace("&#7846;", "Ầ");
            s = s.Replace("&#7848;", "Ẩ");
            s = s.Replace("&#7850;", "Ẫ");
            s = s.Replace("&#7852;", "Ậ");

            s = s.Replace("&#69;", "E");
            s = s.Replace("&#201;", "É");
            s = s.Replace("&#200;", "È");
            s = s.Replace("&#7866;", "Ẻ");
            s = s.Replace("&#7868;", "Ẽ");
            s = s.Replace("&#7864;", "Ẹ");

            s = s.Replace("&#202;", "Ê");
            s = s.Replace("&#7870; ", "Ế");
            s = s.Replace("&#7872;", "Ề");
            s = s.Replace("&#7874;", "Ể");
            s = s.Replace("&#7876;", "Ễ");
            s = s.Replace("&#7878;", "Ệ");

            s = s.Replace("&#73;", "I");
            s = s.Replace("&#205;", "Í");
            s = s.Replace("&#204;", "Ì");
            s = s.Replace("&#7880;", "Ỉ");
            s = s.Replace("&#296;", "Ĩ");
            s = s.Replace("&#7882;", "Ị");

            s = s.Replace("&#79;", "O");
            s = s.Replace("&#211;", "Ó");
            s = s.Replace("&#210;", "Ò");
            s = s.Replace("&#7886;", "Ỏ");
            s = s.Replace("&#213;", "Õ");
            s = s.Replace("&#7884;", "Ọ");

            s = s.Replace("&#212;", "Ô");
            s = s.Replace("&#7888;", "Ố");
            s = s.Replace("&#7890;", "Ồ");
            s = s.Replace("&#7892;", "Ổ");
            s = s.Replace("&#7894;", "Ỗ");
            s = s.Replace("&#7896;", "Ộ");

            s = s.Replace("&#416;", "Ơ");
            s = s.Replace("&#7898;", "Ớ");
            s = s.Replace("&#7900;", "Ờ");
            s = s.Replace("&#7902;", "Ở");
            s = s.Replace("&#7904;", "Ỡ");
            s = s.Replace("&#7906;", "Ợ");

            s = s.Replace("&#85;", "U");
            s = s.Replace("&#218;", "Ú");
            s = s.Replace("&#217;", "Ù");
            s = s.Replace("&#7910;", "Ủ");
            s = s.Replace("&#360;", "Ũ");
            s = s.Replace("&#7908;", "Ụ");

            s = s.Replace("&#431;", "Ư");
            s = s.Replace("&#7912;", "Ứ");
            s = s.Replace("&#7914;", "Ừ");
            s = s.Replace("&#7916;", "Ử");
            s = s.Replace("&#7918;", "Ữ");
            s = s.Replace("&#7920;", "Ự");

            s = s.Replace("&#89;", "Y");
            s = s.Replace("&#221;", "Ý");
            s = s.Replace("&#7922;", "Ỳ");
            s = s.Replace("&#7926;", "Ỷ");
            s = s.Replace("&#7928;", "Ỹ");
            s = s.Replace("&#7924;", "Ỵ");

            s = s.Replace("&#272;", "Đ");

            return s;
        }

        public static string GetWebContent(string strUrl)
        {
            string s = "";
            try
            {
                WebClient wClient = new WebClient();
                wClient.Encoding = Encoding.UTF8;
                string strContent = Encoding.UTF8.GetString(wClient.DownloadData(strUrl));
            }
            catch { s = ""; }
            return s;
        }

        public string GetContentWithStream(string url)
        {
            string s = "";

            try
            {
                WebClient wc = new WebClient();

                wc.Headers.Add("Accept: text/html, application/xhtml+xml, */*");
                wc.Headers.Add("user-agent", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)");
                wc.Headers.Add(HttpRequestHeader.ContentEncoding, "utf-8");

                Stream data = wc.OpenRead(url);
                using (StreamReader reader = new StreamReader(data))
                {
                    s = reader.ReadToEnd();
                    data.Close();
                    reader.Close();
                }
            }
            catch { }

            return s;
        }

        /// <summary>
        /// Hàm này để tạm xóa sau nhé
        /// </summary>
        /// <param name="entity"></param>
        /// <returns></returns>
        private string SaveFileJson(ContentEntity entity)
        {
            try
            {
                var folder = Config.FTP_FOLDER + "/" + entity.Source + "/" + entity.LastModify.ToString("ddMMyyyy");
                if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
                var file = folder + "/" + entity.Id + entity.LastModify.ToUnixTimestamp() + ".txt";

                if (File.Exists(file))
                {
                    return file;
                }

                var sw = new StreamWriter(file);
                sw.Write(entity.ToJson());
                sw.Close();

                /* Tạm tắt, chạy thật mới bật lên */
                if (File.Exists(file))
                {
                    FtpExpress ftp = new FtpExpress(Config.FTP_ADDRESS, Config.FTP_USER, Config.FTP_PASSWORD);
                    bool re = ftp.Upload(file, file, false);
                }

                return file;
            }
            catch (Exception ex)
            {
                return "";
            }
        }

        public string DeleteSameLink()
        { 
            ContentBO conBo = new ContentBO();
            Dictionary<string, long> dic = conBo.GetListToList("http://vnexpress.net");

            if (dic != null && dic.Count != 0)
            {
                foreach (var item in dic)
                {
                    long id = Convert.ToInt64(item.Value);
                    string lnk = item.Key;

                    // xóa các link trùng
                    bool re = conBo.DeleteSameLink(id, lnk);

                    if (re == true)
                    {
                        string s = " ";
                    }
                    else
                    {
                        string s = " ";
                    }
                }
            }

            return "xong";
        }

    }
}
